{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras MNIST Model Deployment\n",
    "\n",
    " * Wrap a Tensorflow MNIST python model for use as a prediction microservice in seldon-core\n",
    "   * Run locally on Docker to test\n",
    "   * Deploy on seldon-core running on minikube\n",
    " \n",
    "## Dependencies\n",
    "\n",
    " * [Helm](https://github.com/kubernetes/helm)\n",
    " * [Minikube](https://github.com/kubernetes/minikube)\n",
    " * [S2I](https://github.com/openshift/source-to-image)\n",
    "\n",
    "```bash\n",
    "pip install seldon-core\n",
    "pip install keras\n",
    "```\n",
    "\n",
    "## Train locally\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import math\n",
    "import datetime\n",
    "#from seldon.pipeline import PipelineSaver\n",
    "import os\n",
    "import tensorflow as tf\n",
    "from keras import backend\n",
    "from keras.models import Model,load_model\n",
    "from keras.layers import Dense,Input\n",
    "from keras.layers import Dropout\n",
    "from keras.layers import Flatten, Reshape\n",
    "from keras.constraints import maxnorm\n",
    "from keras.layers.convolutional import Convolution2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "\n",
    "from keras.callbacks import TensorBoard\n",
    "\n",
    "class MnistFfnn(object):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_shape=(784,),\n",
    "                 nb_labels=10,\n",
    "                 optimizer='Adam',\n",
    "                 run_dir='tensorboardlogs_test'):\n",
    "        \n",
    "        self.model_name='MnistFfnn'\n",
    "        self.run_dir=run_dir\n",
    "        self.input_shape=input_shape\n",
    "        self.nb_labels=nb_labels\n",
    "        self.optimizer=optimizer\n",
    "        self.build_graph()\n",
    "\n",
    "    def build_graph(self):\n",
    "                            \n",
    "        inp = Input(shape=self.input_shape,name='input_part')\n",
    "\n",
    "        #keras layers\n",
    "        with tf.name_scope('dense_1') as scope:\n",
    "            h1 = Dense(256,\n",
    "                         activation='relu',\n",
    "                         W_constraint=maxnorm(3))(inp)\n",
    "            drop1 = Dropout(0.2)(h1)\n",
    "\n",
    "        with tf.name_scope('dense_2') as scope:\n",
    "            h2 = Dense(128,\n",
    "                       activation='relu',\n",
    "                       W_constraint=maxnorm(3))(drop1)\n",
    "            drop2 = Dropout(0.5)(h2)\n",
    "            \n",
    "            out = Dense(self.nb_labels,\n",
    "                        activation='softmax')(drop2)\n",
    "\n",
    "        self.model = Model(inp,out)\n",
    "        \n",
    "        if self.optimizer ==  'rmsprop':\n",
    "            self.model.compile(loss='categorical_crossentropy',\n",
    "                               optimizer='rmsprop',\n",
    "                               metrics=['accuracy'])\n",
    "        elif self.optimizer == 'Adam':\n",
    "            self.model.compile(loss='categorical_crossentropy',\n",
    "                               optimizer='Adam',\n",
    "                               metrics=['accuracy'])\n",
    "            \n",
    "        print('graph builded')\n",
    "\n",
    "    def fit(self,X,y=None,\n",
    "            X_test=None,y_test=None,\n",
    "            batch_size=128,\n",
    "            nb_epochs=2,\n",
    "            shuffle=True):\n",
    "        \n",
    "        now = datetime.datetime.now()\n",
    "        tensorboard_logname = self.run_dir+'/{}_{}'.format(self.model_name,\n",
    "                                                           now.strftime('%Y.%m.%d_%H.%M'))      \n",
    "        tensorboard = TensorBoard(log_dir=tensorboard_logname)\n",
    "        \n",
    "        self.model.fit(X,y,\n",
    "                       validation_data=(X_test,y_test),\n",
    "                       callbacks=[tensorboard],\n",
    "                       batch_size=batch_size, \n",
    "                       nb_epoch=nb_epochs,\n",
    "                       shuffle = shuffle)\n",
    "        return self\n",
    "    \n",
    "    def predict_proba(self,X):\n",
    "\n",
    "        return self.model.predict_proba(X)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        probas = self.model.predict_proba(X)\n",
    "        return([[p>0.5 for p in p1] for p1 in probas])\n",
    "        \n",
    "    def score(self, X, y=None):\n",
    "        pass\n",
    "\n",
    "    def get_class_id_map(self):\n",
    "        return [\"proba\"]\n",
    "\n",
    "class MnistConv(object):\n",
    "\n",
    "    def __init__(self,\n",
    "                 input_shape=(784,),\n",
    "                 nb_labels=10,\n",
    "                 optimizer='Adam',\n",
    "                 run_dir='tensorboardlogs_test',\n",
    "                 saved_model_file='MnistClassifier.h5'):\n",
    "        \n",
    "        self.model_name='MnistConv'\n",
    "        self.run_dir=run_dir\n",
    "        self.input_shape=input_shape\n",
    "        self.nb_labels=nb_labels\n",
    "        self.optimizer=optimizer\n",
    "        self.saved_model_file=saved_model_file\n",
    "        self.build_graph()\n",
    "\n",
    "    def build_graph(self):\n",
    "                                                                \n",
    "        inp = Input(shape=self.input_shape,name='input_part')\n",
    "        inp2 = Reshape((28,28,1))(inp)      \n",
    "        #keras layers\n",
    "        with tf.name_scope('conv') as scope:\n",
    "            conv = Convolution2D(32, 3, 3,\n",
    "                                 input_shape=(32, 32, 3),\n",
    "                                 border_mode='same',\n",
    "                                 activation='relu',\n",
    "                                 W_constraint=maxnorm(3))(inp2)\n",
    "            drop_conv = Dropout(0.2)(conv)\n",
    "            max_pool = MaxPooling2D(pool_size=(2, 2))(drop_conv)\n",
    "\n",
    "        with tf.name_scope('dense') as scope:\n",
    "            flat = Flatten()(max_pool)                \n",
    "            dense = Dense(128,\n",
    "                          activation='relu',\n",
    "                          W_constraint=maxnorm(3))(flat)\n",
    "            drop_dense = Dropout(0.5)(dense)\n",
    "            \n",
    "            out = Dense(self.nb_labels,\n",
    "                        activation='softmax')(drop_dense)\n",
    "\n",
    "        self.model = Model(inp,out)\n",
    "        \n",
    "        if self.optimizer ==  'rmsprop':\n",
    "            self.model.compile(loss='categorical_crossentropy',\n",
    "                               optimizer='rmsprop',\n",
    "                               metrics=['accuracy'])\n",
    "        elif self.optimizer == 'Adam':\n",
    "            self.model.compile(loss='categorical_crossentropy',\n",
    "                               optimizer='Adam',\n",
    "                               metrics=['accuracy'])\n",
    "            \n",
    "        print('graph builded')\n",
    "\n",
    "    def fit(self,X,y=None,\n",
    "            X_test=None,y_test=None,\n",
    "            batch_size=128,\n",
    "            nb_epochs=2,\n",
    "            shuffle=True):\n",
    "        \n",
    "        now = datetime.datetime.now()\n",
    "        tensorboard_logname = self.run_dir+'/{}_{}'.format(self.model_name,\n",
    "                                                           now.strftime('%Y.%m.%d_%H.%M'))      \n",
    "        tensorboard = TensorBoard(log_dir=tensorboard_logname)\n",
    "        \n",
    "        self.model.fit(X,y,\n",
    "                       validation_data=(X_test,y_test),\n",
    "                       callbacks=[tensorboard],\n",
    "                       batch_size=batch_size, \n",
    "                       nb_epoch=nb_epochs,\n",
    "                       shuffle = shuffle)\n",
    "        #if not os.path.exists('saved_model'):\n",
    "        #    os.makedirs('saved_model')\n",
    "        self.model.save(self.saved_model_file)\n",
    "        return self\n",
    "    \n",
    "    def predict_proba(self,X):\n",
    "        return self.model.predict_proba(X)\n",
    "    \n",
    "    def predict(self, X):\n",
    "        probas = self.model.predict_proba(X)\n",
    "        return([[p>0.5 for p in p1] for p1 in probas])\n",
    "        \n",
    "    def score(self, X, y=None):\n",
    "        pass\n",
    "\n",
    "    def get_class_id_map(self):\n",
    "        return [\"proba\"]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-2-dac8c42b25f0>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data/MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting data/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting data/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting data/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/clive/anaconda3/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/clive/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:126: UserWarning: Update your `Conv2D` call to the Keras 2 API: `Conv2D(32, (3, 3), input_shape=(32, 32, 3..., activation=\"relu\", padding=\"same\", kernel_constraint=<keras.con...)`\n",
      "/home/clive/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:134: UserWarning: Update your `Dense` call to the Keras 2 API: `Dense(128, activation=\"relu\", kernel_constraint=<keras.con...)`\n",
      "/home/clive/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:169: UserWarning: The `nb_epoch` argument in `fit` has been renamed `epochs`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "graph builded\n",
      "Train on 55000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "55000/55000 [==============================] - 25s 463us/step - loss: 0.3302 - acc: 0.9025 - val_loss: 0.1015 - val_acc: 0.9727\n",
      "Epoch 2/2\n",
      "55000/55000 [==============================] - 27s 488us/step - loss: 0.1227 - acc: 0.9642 - val_loss: 0.0633 - val_acc: 0.9798\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<__main__.MnistConv at 0x7f048fbf9748>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "mnist = input_data.read_data_sets('data/MNIST_data', one_hot=True)\n",
    "X_train = mnist.train.images\n",
    "y_train = mnist.train.labels\n",
    "X_test = mnist.test.images\n",
    "y_test = mnist.test.labels\n",
    "mc = MnistConv()\n",
    "mc.fit(X_train,y=y_train,\n",
    "    X_test=X_test,y_test=y_test)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Wrap model using s2i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (1.16.3)\n",
      "Collecting scipy>=0.13.3 (from -r requirements.txt (line 2))\n",
      "  WARNING: Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/7f/5f/c48860704092933bf1c4c1574a8de1ffd16bf4fde8bab190d747598844b2/scipy-1.2.1-cp36-cp36m-manylinux1_x86_64.whl (24.8MB)\n",
      "Collecting keras==2.1.3 (from -r requirements.txt (line 3))\n",
      "  WARNING: Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/08/ae/7f94a03cb3f74cdc8a0f5f86d1df5c1dd686acb9a9c2a421c64f8497358e/Keras-2.1.3-py2.py3-none-any.whl (319kB)\n",
      "Requirement already satisfied: tensorflow>=1.12.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 4)) (1.13.1)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (2.9.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/site-packages (from keras==2.1.3->-r requirements.txt (line 3)) (1.12.0)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/site-packages (from keras==2.1.3->-r requirements.txt (line 3)) (5.1)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.7.1)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.7.1)\n",
      "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.13.0)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.0.9)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.20.1)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (3.7.1)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.33.1)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.0.7)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.2.2)\n",
      "Requirement already satisfied: tensorboard<1.14.0,>=1.13.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.13.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.1.0)\n",
      "Requirement already satisfied: mock>=2.0.0 in /usr/local/lib/python3.6/site-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (2.0.0)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/site-packages (from protobuf>=3.6.1->tensorflow>=1.12.0->-r requirements.txt (line 4)) (41.0.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.15.2)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (3.1)\n",
      "Requirement already satisfied: pbr>=0.11 in /usr/local/lib/python3.6/site-packages (from mock>=2.0.0->tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (5.2.0)\n",
      "Installing collected packages: scipy, keras\n",
      "Successfully installed keras-2.1.3 scipy-1.2.1\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!s2i build . seldonio/seldon-core-s2i-python3:0.13 keras-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6231efde4036974469fddd42585db66067e6a30bcfd40efe5cf474d385f0eeda\r\n"
     ]
    }
   ],
   "source": [
    "!docker run --name \"mnist_predictor\" -d --rm -p 5000:5000 keras-mnist:0.1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Send some random features that conform to the contract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[0.387 0.103 0.152 0.129 0.211 0.088 0.659 0.028 0.663 0.666 0.134 0.396\n",
      "  0.704 0.089 0.407 0.896 0.734 0.375 0.109 0.796 0.917 0.186 0.736 0.013\n",
      "  0.565 0.256 0.405 0.205 0.317 0.342 0.02  0.748 0.496 0.376 0.405 0.712\n",
      "  0.775 0.904 0.277 0.973 0.004 0.996 0.692 0.802 0.967 0.361 0.222 0.358\n",
      "  0.73  0.032 0.516 0.945 0.734 0.012 0.807 0.558 0.604 0.978 0.111 0.772\n",
      "  0.276 0.484 0.645 0.73  0.953 0.306 0.049 0.299 0.872 0.197 0.389 0.191\n",
      "  0.604 0.431 0.498 0.091 0.366 0.834 0.266 0.256 0.827 0.996 0.071 0.522\n",
      "  0.108 0.063 0.607 0.126 0.97  0.758 0.99  0.961 0.285 0.547 0.633 0.788\n",
      "  0.619 0.694 0.157 0.91  0.992 0.276 0.422 0.978 0.108 0.272 0.605 0.375\n",
      "  0.964 0.257 0.215 0.583 0.594 0.162 0.118 0.518 0.026 0.687 0.98  0.666\n",
      "  0.233 0.998 0.678 0.379 0.778 0.149 0.889 0.911 0.019 0.183 0.471 0.272\n",
      "  0.513 0.628 0.769 0.062 0.706 0.029 0.31  0.322 0.341 0.492 0.124 0.154\n",
      "  0.643 0.145 0.966 0.874 0.364 0.009 0.611 0.073 0.73  0.712 0.926 0.541\n",
      "  0.96  0.055 0.105 0.869 0.958 0.892 0.437 0.477 0.477 0.09  0.929 0.708\n",
      "  0.839 0.629 0.395 0.878 0.278 0.88  0.078 0.525 0.521 0.292 0.3   0.971\n",
      "  0.002 0.89  0.968 0.19  0.946 0.784 0.926 0.017 0.748 0.287 0.76  0.786\n",
      "  0.201 0.926 0.173 0.399 0.764 0.249 0.228 0.027 0.125 0.271 0.776 0.82\n",
      "  0.007 0.685 0.87  0.997 0.115 0.972 0.439 0.761 0.666 0.793 0.72  0.399\n",
      "  0.361 0.951 0.366 0.942 0.014 0.617 0.634 0.148 0.33  0.943 0.784 0.04\n",
      "  0.514 0.823 0.346 0.428 0.376 0.908 0.584 0.238 0.929 0.149 0.392 0.898\n",
      "  0.358 0.088 0.853 0.016 0.278 0.474 0.892 0.957 0.358 0.058 0.655 0.682\n",
      "  0.32  0.322 0.367 0.069 0.274 0.587 0.662 0.281 0.377 0.281 0.989 0.989\n",
      "  0.787 0.893 0.051 0.839 0.428 0.088 0.62  0.084 0.951 0.663 0.68  0.069\n",
      "  0.208 0.186 0.976 0.657 0.955 0.452 0.429 0.71  0.819 0.091 0.228 0.427\n",
      "  0.995 0.546 0.724 0.022 0.39  0.425 0.871 0.136 0.554 0.383 0.466 0.852\n",
      "  0.673 0.021 0.957 0.573 0.587 0.579 0.149 0.787 0.303 0.484 0.876 0.766\n",
      "  0.167 0.743 0.327 0.486 0.357 0.381 0.403 0.047 0.02  0.823 0.009 0.494\n",
      "  0.919 0.474 0.369 0.364 0.208 0.762 0.942 0.68  0.463 0.369 0.146 0.591\n",
      "  0.028 0.957 0.937 0.133 0.124 0.587 0.506 0.556 0.156 0.078 0.507 0.425\n",
      "  0.634 0.147 0.151 0.278 0.467 0.119 0.682 0.486 0.627 0.599 0.837 0.117\n",
      "  0.686 0.939 0.014 0.801 0.64  0.079 0.811 0.947 0.203 0.294 0.516 0.566\n",
      "  0.237 0.514 0.696 0.121 0.57  0.334 0.206 0.002 0.735 0.951 0.673 0.524\n",
      "  0.548 0.737 0.429 0.141 0.173 0.574 0.024 0.359 0.287 0.467 0.199 0.654\n",
      "  0.682 0.237 0.874 0.909 0.417 0.311 0.764 0.833 0.566 0.207 0.572 0.798\n",
      "  0.852 0.542 0.863 0.626 0.473 0.137 0.582 0.441 0.562 0.939 0.042 0.209\n",
      "  0.763 0.315 0.833 0.923 0.565 0.056 0.002 0.833 0.157 0.905 0.221 0.993\n",
      "  0.863 0.312 0.752 0.875 0.02  0.54  0.96  0.901 0.831 0.384 0.589 0.704\n",
      "  0.161 0.591 0.068 0.656 0.713 0.347 0.775 0.903 0.137 0.846 0.497 0.394\n",
      "  0.4   0.264 0.095 0.839 0.746 0.955 0.843 0.352 0.413 0.531 0.83  0.176\n",
      "  0.89  0.114 0.06  0.012 0.56  0.443 0.78  0.946 0.922 0.178 0.76  0.547\n",
      "  0.683 0.418 0.83  0.773 0.434 0.236 0.503 0.759 0.102 0.243 0.33  0.054\n",
      "  0.629 0.014 0.612 0.257 0.281 0.519 0.619 0.385 0.759 0.513 0.753 0.862\n",
      "  0.318 0.002 0.114 0.457 0.568 0.006 0.019 0.722 0.328 0.135 0.353 0.856\n",
      "  0.434 0.839 0.123 0.864 0.173 0.307 0.711 0.767 0.528 0.2   0.195 0.854\n",
      "  0.993 0.374 0.804 0.389 0.248 0.208 0.437 0.806 0.7   0.16  0.548 0.628\n",
      "  0.768 0.278 0.62  0.17  0.603 0.716 0.294 0.426 0.655 0.373 0.229 0.666\n",
      "  0.464 0.437 0.598 0.553 0.06  0.342 0.541 0.677 0.03  0.02  0.576 0.85\n",
      "  0.829 0.696 0.069 0.321 0.945 0.218 0.768 0.84  0.735 0.93  0.107 0.962\n",
      "  0.883 0.106 0.348 0.081 0.335 0.037 0.595 0.083 0.457 0.382 0.825 0.614\n",
      "  0.925 0.959 0.689 0.988 0.604 0.937 0.8   0.191 0.633 0.744 0.999 0.812\n",
      "  0.883 0.31  0.745 0.344 0.086 0.257 0.315 0.411 0.694 0.296 0.257 0.84\n",
      "  0.381 0.237 0.28  0.842 0.535 0.439 0.191 0.814 0.224 0.813 0.901 0.797\n",
      "  0.855 0.86  0.106 0.763 0.137 0.055 0.08  0.515 0.578 0.892 0.311 0.522\n",
      "  0.31  0.145 0.171 0.684 0.682 0.577 0.294 0.278 0.485 0.867 0.205 0.483\n",
      "  0.405 0.728 0.596 0.584 0.4   0.276 0.707 0.398 0.16  0.551 0.362 0.471\n",
      "  0.031 0.125 0.254 0.224 0.091 0.948 0.941 0.383 0.506 0.324 0.125 0.049\n",
      "  0.148 0.168 0.269 0.818 0.69  0.936 0.234 0.336 0.718 0.929 0.908 0.596\n",
      "  0.208 0.042 0.657 0.26  0.577 0.691 0.953 0.193 0.772 0.245 0.296 0.527\n",
      "  0.262 0.545 0.394 0.899 0.975 0.824 0.877 0.933 0.725 0.035 0.496 0.102\n",
      "  0.313 0.287 0.894 0.046 0.574 0.766 0.761 0.493 0.25  0.454 0.475 0.272\n",
      "  0.838 0.843 0.595 0.182 0.497 0.049 0.294 0.926 0.018 0.448 0.494 0.008\n",
      "  0.667 0.392 0.659 0.703 0.113 0.435 0.411 0.011 0.851 0.214 0.364 0.074\n",
      "  0.279 0.743 0.49  0.183 0.157 0.263 0.669 0.583 0.406 0.81  0.093 0.562\n",
      "  0.525 0.631 0.786 0.74  0.156 0.797 0.251 0.599 0.959 0.553 0.343 0.167\n",
      "  0.729 0.814 0.368 0.616 0.946 0.036 0.889 0.112 0.584 0.462 0.673 0.082\n",
      "  0.538 0.901 0.973 0.161]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  names: \"t:4\"\n",
      "  names: \"t:5\"\n",
      "  names: \"t:6\"\n",
      "  names: \"t:7\"\n",
      "  names: \"t:8\"\n",
      "  names: \"t:9\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 0.00022297287068795413\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.003534407587721944\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.1571815013885498\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.22603441774845123\n",
      "        }\n",
      "        values {\n",
      "          number_value: 5.994380626361817e-05\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0454179011285305\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.34811070561408997\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.21694059669971466\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0018390808254480362\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0006583957583643496\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-tester contract.json 0.0.0.0 5000 -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mnist_predictor\r\n"
     ]
    }
   ],
   "source": [
    "!docker rm mnist_predictor --force"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test using Minikube\n",
    "\n",
    "**Due to a [minikube/s2i issue](https://github.com/SeldonIO/seldon-core/issues/253) you will need [s2i >= 1.1.13](https://github.com/openshift/source-to-image/releases/tag/v1.1.13)**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "😄  minikube v1.0.0 on linux (amd64)\n",
      "🤹  Downloading Kubernetes v1.14.0 images in the background ...\n",
      "🔥  Creating virtualbox VM (CPUs=2, Memory=4096MB, Disk=20000MB) ...\n",
      "📶  \"minikube\" IP address is 192.168.99.100\n",
      "🐳  Configuring Docker as the container runtime ...\n",
      "🐳  Version of container runtime is 18.06.2-ce\n",
      "⌛  Waiting for image downloads to complete ...\n",
      "✨  Preparing Kubernetes environment ...\n",
      "🚜  Pulling images required by Kubernetes v1.14.0 ...\n",
      "🚀  Launching Kubernetes v1.14.0 using kubeadm ... \n",
      "⌛  Waiting for pods: apiserver proxy etcd scheduler controller dns\n",
      "🔑  Configuring cluster permissions ...\n",
      "🤔  Verifying component health .....\n",
      "💗  kubectl is now configured to use \"minikube\"\n",
      "🏄  Done! Thank you for using minikube!\n"
     ]
    }
   ],
   "source": [
    "!minikube start --memory 4096 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clusterrolebinding.rbac.authorization.k8s.io/kube-system-cluster-admin created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create clusterrolebinding kube-system-cluster-admin --clusterrole=cluster-admin --serviceaccount=kube-system:default"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "$HELM_HOME has been configured at /home/clive/.helm.\n",
      "\n",
      "Tiller (the Helm server-side component) has been installed into your Kubernetes Cluster.\n",
      "\n",
      "Please note: by default, Tiller is deployed with an insecure 'allow unauthenticated users' policy.\n",
      "To prevent this, run `helm init` with the --tiller-tls-verify flag.\n",
      "For more information on securing your installation see: https://docs.helm.sh/using_helm/#securing-your-helm-installation\n",
      "Happy Helming!\n"
     ]
    }
   ],
   "source": [
    "!helm init"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"tiller-deploy\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"tiller-deploy\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/tiller-deploy -n kube-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   seldon-core\n",
      "LAST DEPLOYED: Fri May  3 19:38:47 2019\n",
      "NAMESPACE: seldon-system\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/ClusterRoleBinding\n",
      "NAME                                 AGE\n",
      "seldon-operator-manager-rolebinding  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME                                        TYPE       CLUSTER-IP      EXTERNAL-IP  PORT(S)  AGE\n",
      "seldon-operator-controller-manager-service  ClusterIP  10.101.135.115  <none>       443/TCP  0s\n",
      "\n",
      "==> v1/StatefulSet\n",
      "NAME                                DESIRED  CURRENT  AGE\n",
      "seldon-operator-controller-manager  1        1        0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                                  READY  STATUS             RESTARTS  AGE\n",
      "seldon-operator-controller-manager-0  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/Secret\n",
      "NAME                                   TYPE    DATA  AGE\n",
      "seldon-operator-webhook-server-secret  Opaque  0     0s\n",
      "\n",
      "==> v1beta1/CustomResourceDefinition\n",
      "NAME                                         AGE\n",
      "seldondeployments.machinelearning.seldon.io  0s\n",
      "\n",
      "==> v1/ClusterRole\n",
      "seldon-operator-manager-role  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "NOTES: TODO\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install ../../../helm-charts/seldon-core-operator --name seldon-core --set usageMetrics.enabled=true   --namespace seldon-system"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for 1 pods to be ready...\n",
      "partitioned roll out complete: 1 new pods have been updated...\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/seldon-controller-manager -n seldon-system"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup Ingress\n",
    "Please note: There are reported gRPC issues with ambassador (see https://github.com/SeldonIO/seldon-core/issues/473)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NAME:   ambassador\n",
      "LAST DEPLOYED: Fri May  3 19:40:12 2019\n",
      "NAMESPACE: default\n",
      "STATUS: DEPLOYED\n",
      "\n",
      "RESOURCES:\n",
      "==> v1/Deployment\n",
      "NAME        DESIRED  CURRENT  UP-TO-DATE  AVAILABLE  AGE\n",
      "ambassador  3        3        3           0          0s\n",
      "\n",
      "==> v1/Pod(related)\n",
      "NAME                         READY  STATUS             RESTARTS  AGE\n",
      "ambassador-5b89d44544-f7prg  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-jkbtz  0/1    ContainerCreating  0         0s\n",
      "ambassador-5b89d44544-mjgj8  0/1    ContainerCreating  0         0s\n",
      "\n",
      "==> v1/ServiceAccount\n",
      "NAME        SECRETS  AGE\n",
      "ambassador  1        0s\n",
      "\n",
      "==> v1beta1/ClusterRole\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1beta1/ClusterRoleBinding\n",
      "NAME        AGE\n",
      "ambassador  0s\n",
      "\n",
      "==> v1/Service\n",
      "NAME               TYPE          CLUSTER-IP      EXTERNAL-IP  PORT(S)                     AGE\n",
      "ambassador-admins  ClusterIP     10.100.85.59    <none>       8877/TCP                    0s\n",
      "ambassador         LoadBalancer  10.109.230.160  <pending>    80:31635/TCP,443:30969/TCP  0s\n",
      "\n",
      "\n",
      "NOTES:\n",
      "Congratuations! You've successfully installed Ambassador.\n",
      "\n",
      "For help, visit our Slack at https://d6e.co/slack or view the documentation online at https://www.getambassador.io.\n",
      "\n",
      "To get the IP address of Ambassador, run the following commands:\n",
      "NOTE: It may take a few minutes for the LoadBalancer IP to be available.\n",
      "     You can watch the status of by running 'kubectl get svc -w  --namespace default ambassador'\n",
      "\n",
      "  On GKE/Azure:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].ip}')\n",
      "\n",
      "  On AWS:\n",
      "  export SERVICE_IP=$(kubectl get svc --namespace default ambassador -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')\n",
      "\n",
      "  echo http://$SERVICE_IP:\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!helm install stable/ambassador --name ambassador --set crds.keep=false"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"ambassador\" rollout to finish: 0 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 1 of 3 updated replicas are available...\n",
      "Waiting for deployment \"ambassador\" rollout to finish: 2 of 3 updated replicas are available...\n",
      "deployment \"ambassador\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deployment.apps/ambassador"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---> Installing application source...\n",
      "---> Installing dependencies ...\n",
      "Looking in links: /whl\n",
      "Requirement already satisfied: numpy>=1.8.2 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 1)) (1.16.3)\n",
      "Collecting scipy>=0.13.3 (from -r requirements.txt (line 2))\n",
      "  WARNING: Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/7f/5f/c48860704092933bf1c4c1574a8de1ffd16bf4fde8bab190d747598844b2/scipy-1.2.1-cp36-cp36m-manylinux1_x86_64.whl (24.8MB)\n",
      "Collecting keras==2.1.3 (from -r requirements.txt (line 3))\n",
      "  WARNING: Url '/whl' is ignored. It is either a non-existing path or lacks a specific scheme.\n",
      "Downloading https://files.pythonhosted.org/packages/08/ae/7f94a03cb3f74cdc8a0f5f86d1df5c1dd686acb9a9c2a421c64f8497358e/Keras-2.1.3-py2.py3-none-any.whl (319kB)\n",
      "Requirement already satisfied: tensorflow>=1.12.0 in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 4)) (1.13.1)\n",
      "Requirement already satisfied: h5py in /usr/local/lib/python3.6/site-packages (from -r requirements.txt (line 5)) (2.9.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/local/lib/python3.6/site-packages (from keras==2.1.3->-r requirements.txt (line 3)) (1.12.0)\n",
      "Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/site-packages (from keras==2.1.3->-r requirements.txt (line 3)) (5.1)\n",
      "Requirement already satisfied: gast>=0.2.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.2.2)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.0.7)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.1.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.33.1)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.7.1)\n",
      "Requirement already satisfied: tensorboard<1.14.0,>=1.13.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.13.1)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.0.9)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.20.1)\n",
      "Requirement already satisfied: astor>=0.6.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.7.1)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (3.7.1)\n",
      "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /usr/local/lib/python3.6/site-packages (from tensorflow>=1.12.0->-r requirements.txt (line 4)) (1.13.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (3.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /usr/local/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (0.15.2)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/site-packages (from protobuf>=3.6.1->tensorflow>=1.12.0->-r requirements.txt (line 4)) (41.0.1)\n",
      "Requirement already satisfied: mock>=2.0.0 in /usr/local/lib/python3.6/site-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (2.0.0)\n",
      "Requirement already satisfied: pbr>=0.11 in /usr/local/lib/python3.6/site-packages (from mock>=2.0.0->tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow>=1.12.0->-r requirements.txt (line 4)) (5.2.0)\n",
      "Installing collected packages: scipy, keras\n",
      "Successfully installed keras-2.1.3 scipy-1.2.1\n",
      "Build completed successfully\n"
     ]
    }
   ],
   "source": [
    "!eval $(minikube docker-env) && s2i build . seldonio/seldon-core-s2i-python3:0.13 keras-mnist:0.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seldondeployment.machinelearning.seldon.io/seldon-deployment-example created\r\n"
     ]
    }
   ],
   "source": [
    "!kubectl create -f keras_mnist_deployment.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Waiting for deployment \"keras-mnist-deployment-keras-mnist-predictor-8baf5cc\" rollout to finish: 0 of 1 updated replicas are available...\n",
      "deployment \"keras-mnist-deployment-keras-mnist-predictor-8baf5cc\" successfully rolled out\n"
     ]
    }
   ],
   "source": [
    "!kubectl rollout status deploy/keras-mnist-deployment-keras-mnist-predictor-8baf5cc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------\n",
      "SENDING NEW REQUEST:\n",
      "\n",
      "[[0.615 0.937 0.603 0.929 0.9   0.267 0.498 0.514 0.13  0.579 0.213 0.063\n",
      "  0.671 0.524 0.455 0.049 0.159 0.379 0.886 0.302 0.024 0.57  0.86  0.979\n",
      "  0.908 0.502 0.427 0.818 0.711 0.83  0.496 0.908 0.567 0.065 0.639 0.464\n",
      "  0.699 0.415 0.356 0.181 0.152 0.409 0.901 0.981 0.648 0.761 0.721 0.867\n",
      "  0.76  0.834 0.092 0.236 0.881 0.292 0.229 0.37  0.069 0.413 0.007 0.15\n",
      "  0.132 0.851 0.75  0.026 0.614 0.533 0.082 0.805 0.176 0.662 0.379 0.002\n",
      "  0.001 0.132 0.345 0.016 0.317 0.418 0.197 0.846 0.956 0.193 0.447 0.835\n",
      "  0.2   0.313 0.094 0.94  0.068 0.724 0.732 0.561 0.763 0.589 0.056 0.893\n",
      "  0.867 0.548 0.365 0.865 0.459 0.217 0.686 0.831 0.952 0.526 0.567 0.544\n",
      "  0.84  0.642 0.659 0.266 0.666 0.401 0.77  0.646 0.477 0.646 0.186 0.39\n",
      "  0.197 0.216 0.552 0.465 0.294 0.596 0.955 0.117 0.644 0.31  0.925 0.559\n",
      "  0.113 0.897 0.379 0.307 0.581 0.044 0.644 0.31  0.871 0.001 0.266 0.356\n",
      "  0.17  0.16  0.761 0.035 0.217 0.417 0.877 0.862 0.199 0.39  0.135 0.795\n",
      "  0.181 0.587 0.597 0.498 0.711 0.957 0.033 0.238 0.964 0.112 0.292 0.703\n",
      "  0.797 0.931 0.582 0.438 0.56  0.599 0.015 0.438 0.807 0.013 0.812 0.841\n",
      "  0.604 0.763 0.084 0.325 0.854 0.523 0.562 0.112 0.296 0.674 0.173 0.323\n",
      "  0.077 0.743 0.396 0.848 0.666 0.505 0.319 0.366 0.345 0.394 0.656 0.124\n",
      "  0.86  0.301 0.481 0.547 0.552 0.769 0.812 0.241 0.476 0.513 0.946 0.877\n",
      "  0.415 0.396 0.553 0.261 0.987 0.157 0.417 0.311 0.    0.49  0.315 0.051\n",
      "  0.847 0.848 0.595 0.707 0.536 0.604 0.844 0.149 0.499 0.763 0.474 0.795\n",
      "  0.95  0.829 0.193 0.032 0.753 0.15  0.759 0.269 0.186 0.084 0.373 0.728\n",
      "  0.316 0.919 0.052 0.722 0.434 0.314 0.215 0.394 0.814 0.973 0.08  0.378\n",
      "  0.47  0.29  0.626 0.737 0.714 0.04  0.243 0.474 0.649 0.812 0.565 0.544\n",
      "  0.716 0.521 0.831 0.638 0.294 0.441 0.55  0.975 0.409 0.126 0.767 0.761\n",
      "  0.121 0.998 0.389 0.057 0.009 0.933 0.947 0.727 0.376 0.297 0.108 0.104\n",
      "  0.516 0.014 0.844 0.044 0.804 0.292 0.985 0.933 0.091 0.643 0.238 0.249\n",
      "  0.023 0.065 0.65  0.453 0.658 0.053 0.725 0.939 0.732 0.65  0.59  0.835\n",
      "  0.537 0.829 0.649 0.414 0.396 0.805 0.46  0.098 0.707 0.39  0.441 0.567\n",
      "  0.961 0.361 0.203 0.04  0.528 0.553 0.591 0.682 0.137 0.661 0.401 0.132\n",
      "  0.16  0.421 0.13  0.567 0.054 0.802 0.784 0.301 0.708 0.409 0.171 0.745\n",
      "  0.5   0.475 0.813 0.397 0.486 0.406 0.841 0.792 0.77  0.216 0.122 0.126\n",
      "  0.774 0.458 0.363 0.233 0.765 0.616 0.473 0.029 0.16  0.09  0.699 0.19\n",
      "  0.891 0.857 0.626 0.132 0.994 0.415 0.53  0.918 0.095 0.589 0.685 0.184\n",
      "  0.355 0.829 0.519 0.164 0.636 0.042 0.869 0.507 0.756 0.535 0.72  0.881\n",
      "  0.526 0.632 0.583 0.838 0.838 0.917 0.369 0.466 0.884 0.015 0.591 0.214\n",
      "  0.053 0.944 0.946 0.652 0.341 0.697 0.229 0.35  0.275 0.729 0.856 0.066\n",
      "  0.196 0.095 0.421 0.462 0.769 0.424 0.704 0.963 0.446 0.574 0.379 0.151\n",
      "  0.247 0.04  0.325 0.853 0.046 0.286 0.85  0.909 0.236 0.867 0.022 0.936\n",
      "  0.039 0.096 0.062 0.516 0.317 0.551 0.058 0.504 0.503 0.795 0.576 0.093\n",
      "  0.529 0.409 0.37  0.444 0.113 0.113 0.037 0.967 0.278 0.339 0.343 0.979\n",
      "  0.341 0.843 0.836 0.678 0.775 0.935 0.104 0.505 0.839 0.421 0.838 0.644\n",
      "  0.078 0.365 0.521 0.754 0.511 0.195 0.606 0.088 0.932 0.74  0.703 0.132\n",
      "  0.172 0.962 0.261 0.786 0.642 0.207 0.899 0.405 0.493 0.58  0.541 0.684\n",
      "  0.616 0.878 0.39  0.834 0.505 0.765 0.292 0.658 0.373 0.961 0.69  0.094\n",
      "  0.318 0.657 0.466 0.58  0.975 0.559 0.114 0.667 0.46  0.719 0.447 0.383\n",
      "  0.106 0.55  0.331 0.614 0.47  0.107 0.939 0.526 0.32  0.667 0.064 0.738\n",
      "  0.755 0.598 0.96  0.268 0.646 0.774 0.951 0.519 0.645 0.767 0.188 0.003\n",
      "  0.202 0.962 0.272 0.798 0.278 0.072 0.128 0.629 0.025 0.78  0.911 0.335\n",
      "  0.178 0.854 0.568 0.276 0.76  0.52  0.55  0.934 0.735 0.421 0.805 0.979\n",
      "  0.039 0.711 0.144 0.685 0.655 0.913 0.621 0.848 0.397 0.249 0.825 0.336\n",
      "  0.601 0.631 0.868 0.54  0.788 0.542 0.588 0.036 0.01  0.412 0.114 0.244\n",
      "  0.026 0.362 0.551 0.982 0.508 0.718 0.889 0.701 0.385 0.701 0.183 0.694\n",
      "  0.238 0.745 0.749 0.595 0.835 0.495 0.018 0.698 0.36  0.64  0.723 0.724\n",
      "  0.417 0.962 0.857 0.908 0.308 0.011 0.397 0.599 0.443 0.399 0.224 0.973\n",
      "  0.69  0.254 0.777 0.756 0.91  0.973 0.999 0.17  0.824 0.087 0.238 0.821\n",
      "  0.96  0.336 0.922 0.822 0.595 0.439 0.311 0.304 0.389 0.835 0.904 0.408\n",
      "  0.992 0.593 0.906 0.84  0.749 0.706 0.401 0.86  0.137 0.559 0.205 0.948\n",
      "  0.446 0.58  0.762 0.738 0.566 0.149 0.725 0.238 0.484 0.027 0.758 0.409\n",
      "  0.98  0.028 0.433 0.911 0.893 0.346 0.502 0.311 0.154 0.606 0.979 0.89\n",
      "  0.276 0.388 0.404 0.666 0.273 0.088 0.193 0.557 0.009 0.293 0.479 0.3\n",
      "  0.919 0.212 0.119 0.669 0.893 0.926 0.853 0.671 0.739 0.007 0.241 0.633\n",
      "  0.185 0.709 0.99  0.175 0.623 0.523 0.864 0.948 0.779 0.161 0.645 0.778\n",
      "  0.377 0.593 0.531 0.668 0.551 0.363 0.798 0.444 0.808 0.691 0.15  0.915\n",
      "  0.502 0.858 0.373 0.568 0.301 0.339 0.035 0.333 0.763 0.789 0.541 0.964\n",
      "  0.578 0.575 0.875 0.267 0.128 0.64  0.068 0.633 0.723 0.19  0.768 0.446\n",
      "  0.387 0.946 0.366 0.947]]\n",
      "RECEIVED RESPONSE:\n",
      "meta {\n",
      "  puid: \"8t8gotatm360hcu7ldv5s9goeo\"\n",
      "  requestPath {\n",
      "    key: \"keras-mnist-classifier\"\n",
      "    value: \"keras-mnist:0.1\"\n",
      "  }\n",
      "}\n",
      "data {\n",
      "  names: \"t:0\"\n",
      "  names: \"t:1\"\n",
      "  names: \"t:2\"\n",
      "  names: \"t:3\"\n",
      "  names: \"t:4\"\n",
      "  names: \"t:5\"\n",
      "  names: \"t:6\"\n",
      "  names: \"t:7\"\n",
      "  names: \"t:8\"\n",
      "  names: \"t:9\"\n",
      "  ndarray {\n",
      "    values {\n",
      "      list_value {\n",
      "        values {\n",
      "          number_value: 1.9628670997917652e-05\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.000876674719620496\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.011045475490391254\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.39959368109703064\n",
      "        }\n",
      "        values {\n",
      "          number_value: 1.71219180629123e-05\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.24513600766658783\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.024894580245018005\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.31388890743255615\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.00057043950073421\n",
      "        }\n",
      "        values {\n",
      "          number_value: 0.0039573488757014275\n",
      "        }\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!seldon-core-api-tester contract.json `minikube ip` `kubectl get svc ambassador -o jsonpath='{.spec.ports[0].nodePort}'` \\\n",
    "    seldon-deployment-example --namespace default -p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!minikube delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
